{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c5ab816f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#5.2. 参数管理\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "94afe4b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0197],\n",
       "        [-0.0074]], grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "net = nn.Sequential(nn.Linear(4, 8), nn.ReLU(), nn.Linear(8, 1))\n",
    "X = torch.rand(size=(2, 4))\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fb6fd5be",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('weight', tensor([[-0.2725, -0.2318, -0.0561, -0.1345,  0.3441,  0.2219, -0.0402, -0.0553]])), ('bias', tensor([0.0146]))])\n"
     ]
    }
   ],
   "source": [
    "print(net[2].state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5c7b23c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "  (1): ReLU()\n",
      "  (2): Linear(in_features=8, out_features=1, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "754c302d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('weight', tensor([[-0.3922, -0.1884, -0.3822, -0.4490],\n",
      "        [ 0.1534,  0.4173,  0.1832, -0.1909],\n",
      "        [ 0.3913, -0.4789,  0.1673, -0.0129],\n",
      "        [-0.0462,  0.1933,  0.2290, -0.3998],\n",
      "        [-0.4960,  0.2906, -0.4256,  0.0369],\n",
      "        [-0.0071,  0.4035, -0.4652, -0.0756],\n",
      "        [ 0.2753, -0.0796, -0.3796,  0.2073],\n",
      "        [ 0.2349, -0.4468,  0.0231, -0.0589]])), ('bias', tensor([ 0.1882, -0.4061,  0.0854, -0.0282, -0.1170, -0.0801, -0.2565, -0.4851]))])\n"
     ]
    }
   ],
   "source": [
    "print(net[0].state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "611e0564",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.nn.parameter.Parameter'>\n",
      "Parameter containing:\n",
      "tensor([0.0146], requires_grad=True)\n",
      "tensor([0.0146])\n"
     ]
    }
   ],
   "source": [
    "print(type(net[2].bias))\n",
    "print(net[2].bias)\n",
    "print(net[2].bias.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "db30ecd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[2].weight.grad == None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "81500412",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('weight', torch.Size([8, 4])) ('bias', torch.Size([8]))\n",
      "('0.weight', torch.Size([8, 4])) ('0.bias', torch.Size([8])) ('2.weight', torch.Size([1, 8])) ('2.bias', torch.Size([1]))\n"
     ]
    }
   ],
   "source": [
    "print(*[(name, param.shape) for name, param in net[0].named_parameters()])\n",
    "print(*[(name, param.shape) for name, param in net.named_parameters()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b2bda69d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.3922, -0.1884, -0.3822, -0.4490],\n",
       "        [ 0.1534,  0.4173,  0.1832, -0.1909],\n",
       "        [ 0.3913, -0.4789,  0.1673, -0.0129],\n",
       "        [-0.0462,  0.1933,  0.2290, -0.3998],\n",
       "        [-0.4960,  0.2906, -0.4256,  0.0369],\n",
       "        [-0.0071,  0.4035, -0.4652, -0.0756],\n",
       "        [ 0.2753, -0.0796, -0.3796,  0.2073],\n",
       "        [ 0.2349, -0.4468,  0.0231, -0.0589]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.state_dict()['0.weight'].data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "07f4a32a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0389],\n",
       "        [-0.0390]], grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def block1():\n",
    "    return nn.Sequential(nn.Linear(4, 8), nn.ReLU(),\n",
    "                         nn.Linear(8, 4), nn.ReLU())\n",
    "\n",
    "def block2():\n",
    "    net = nn.Sequential()\n",
    "    for i in range(4):\n",
    "        # 在这里嵌套\n",
    "        net.add_module(f'block {i}', block1())\n",
    "    return net\n",
    "\n",
    "rgnet = nn.Sequential(block2(), nn.Linear(4, 1))\n",
    "rgnet(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5e7ecd50",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Sequential(\n",
      "    (block 0): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block 1): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block 2): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block 3): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "  )\n",
      "  (1): Linear(in_features=4, out_features=1, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(rgnet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "77746194",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.0298])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rgnet[1].bias.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c7b3136e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('0.block 0.0.weight', torch.Size([8, 4])) ('0.block 0.0.bias', torch.Size([8])) ('0.block 0.2.weight', torch.Size([4, 8])) ('0.block 0.2.bias', torch.Size([4])) ('0.block 1.0.weight', torch.Size([8, 4])) ('0.block 1.0.bias', torch.Size([8])) ('0.block 1.2.weight', torch.Size([4, 8])) ('0.block 1.2.bias', torch.Size([4])) ('0.block 2.0.weight', torch.Size([8, 4])) ('0.block 2.0.bias', torch.Size([8])) ('0.block 2.2.weight', torch.Size([4, 8])) ('0.block 2.2.bias', torch.Size([4])) ('0.block 3.0.weight', torch.Size([8, 4])) ('0.block 3.0.bias', torch.Size([8])) ('0.block 3.2.weight', torch.Size([4, 8])) ('0.block 3.2.bias', torch.Size([4])) ('1.weight', torch.Size([1, 4])) ('1.bias', torch.Size([1]))\n"
     ]
    }
   ],
   "source": [
    "print(*[(name, param.shape) for name, param in rgnet.named_parameters()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f13a6c2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 0.1226,  0.1338,  0.0348, -0.0729])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rgnet.state_dict()['0.block 3.2.bias'].data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0d7b5896",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('0.block 0.0.weight',\n",
       " Parameter containing:\n",
       " tensor([[-0.3984,  0.0446,  0.2945, -0.0960],\n",
       "         [ 0.0976, -0.3359, -0.2903, -0.0018],\n",
       "         [-0.3287, -0.0853, -0.0284, -0.2521],\n",
       "         [ 0.4615, -0.3293, -0.1275, -0.1013],\n",
       "         [-0.4531,  0.0417, -0.3478,  0.0621],\n",
       "         [-0.2971, -0.1690, -0.4568,  0.2903],\n",
       "         [ 0.2873, -0.2650, -0.1610, -0.1706],\n",
       "         [-0.1938,  0.4225,  0.1105, -0.2574]], requires_grad=True))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a  = rgnet.named_parameters()\n",
    "a.__next__()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d84a9e9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('0.block 0.0.bias',\n",
       " Parameter containing:\n",
       " tensor([-0.0475,  0.0881,  0.4417,  0.3686, -0.1790, -0.1822,  0.1821,  0.0711],\n",
       "        requires_grad=True))"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.__next__()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "27270042",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('0.block 0.2.weight',\n",
       " Parameter containing:\n",
       " tensor([[-0.0170, -0.2136, -0.0990,  0.1775,  0.0034,  0.0473,  0.1962, -0.1026],\n",
       "         [ 0.0894,  0.0078,  0.2680, -0.1514,  0.1634,  0.0749,  0.0490, -0.3024],\n",
       "         [-0.0576,  0.1218,  0.2725, -0.1255, -0.2657,  0.1430, -0.0289, -0.1965],\n",
       "         [-0.0212,  0.0997,  0.1960,  0.2604, -0.1103, -0.0531, -0.2636,  0.1782]],\n",
       "        requires_grad=True))"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.__next__()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "4a6b0b43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([-0.0082,  0.0203, -0.0209, -0.0105]), tensor(0.))"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def init_normal(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.normal_(m.weight, mean=0, std=0.01)\n",
    "        nn.init.zeros_(m.bias)\n",
    "net.apply(init_normal)\n",
    "net[0].weight.data[0], net[0].bias.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "6cc50e9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([1., 1., 1., 1.]), tensor(0.))"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def init_constant(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.constant_(m.weight, 1)\n",
    "        nn.init.zeros_(m.bias)\n",
    "net.apply(init_constant)\n",
    "net[0].weight.data[0], net[0].bias.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b10ccba0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-0.0477, -0.1571,  0.4873, -0.4309])\n",
      "tensor([[42., 42., 42., 42., 42., 42., 42., 42.]])\n"
     ]
    }
   ],
   "source": [
    "def xavier(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.xavier_uniform_(m.weight)\n",
    "def init_42(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.constant_(m.weight, 42)\n",
    "\n",
    "net[0].apply(xavier)\n",
    "net[2].apply(init_42)\n",
    "print(net[0].weight.data[0])\n",
    "print(net[2].weight.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "418f31a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "  (1): ReLU()\n",
      "  (2): Linear(in_features=8, out_features=1, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a681c90a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Init weight Parameter containing:\n",
      "tensor([[-0.0000, -0.0000, -6.4143,  6.6585],\n",
      "        [ 8.5735,  0.0000,  0.0000,  0.0000],\n",
      "        [ 5.2431,  8.4718,  5.4644, -8.5105],\n",
      "        [-0.0000,  0.0000, -8.1288,  8.8649],\n",
      "        [-0.0000, -0.0000, -5.2037,  0.0000],\n",
      "        [-0.0000, -8.8613,  5.2307,  7.1947],\n",
      "        [ 0.0000,  7.2318,  8.6716, -7.9558],\n",
      "        [ 0.0000,  5.9993, -0.0000, -0.0000]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 8.5029,  7.3467, -0.5140, -8.4980],\n",
      "        [ 9.3236, -2.0277,  9.1272,  2.2480],\n",
      "        [-7.0872,  2.9957, -1.6593, -0.8622],\n",
      "        [-7.7967,  1.7040,  9.1982, -3.4324],\n",
      "        [-2.6501, -4.7111, -3.6101, -7.3363],\n",
      "        [ 5.5021,  9.3390,  7.4439, -6.5267],\n",
      "        [ 8.2948,  8.4001, -8.6048, -6.7996],\n",
      "        [ 6.5498,  5.5113,  6.3609,  9.4649]], requires_grad=True)\n",
      "Init weight Parameter containing:\n",
      "tensor([[-8.8112,  0.0000, -6.6966,  5.8764, -0.0000, -0.0000, -0.0000, -0.0000]],\n",
      "       requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[ 9.4951,  3.6154, -6.9261,  4.4132, -5.7013, -2.3005, -0.0156, -9.4688]],\n",
      "       requires_grad=True)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 9.4951,  0.0000, -6.9261,  0.0000, -5.7013, -0.0000, -0.0000, -9.4688]],\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def my_init(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        print(\"Init\", *[(name, param)\n",
    "                        for name, param in m.named_parameters()][0])\n",
    "        nn.init.uniform_(m.weight, -10, 10)\n",
    "        print(m.weight)\n",
    "        m.weight.data *= m.weight.data.abs() >= 5\n",
    "\n",
    "net.apply(my_init)\n",
    "net[2].weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "59edde77",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "  (1): ReLU()\n",
      "  (2): Linear(in_features=8, out_features=8, bias=True)\n",
      "  (3): ReLU()\n",
      "  (4): Linear(in_features=8, out_features=8, bias=True)\n",
      "  (5): ReLU()\n",
      "  (6): Linear(in_features=8, out_features=1, bias=True)\n",
      ")\n",
      "tensor([True, True, True, True, True, True, True, True])\n",
      "Parameter containing:\n",
      "tensor([[ 1.0100e+02, -5.3551e-02, -2.2344e-01,  1.0967e-01,  3.3730e-01,\n",
      "          2.7615e-01, -1.0787e-01, -2.0000e-01],\n",
      "        [ 1.8817e-01,  2.5343e-01,  3.0534e-01, -3.3679e-02, -1.0514e-01,\n",
      "         -1.4801e-01, -1.7283e-01, -5.0863e-02],\n",
      "        [ 2.7657e-01, -1.2663e-01, -3.4786e-01, -3.3446e-01, -7.9900e-02,\n",
      "          2.8380e-01, -1.7702e-03,  3.4718e-01],\n",
      "        [ 3.2893e-01, -3.8217e-02,  2.1321e-01, -1.0167e-01,  6.1079e-02,\n",
      "         -2.5851e-01, -2.5678e-01, -1.0435e-01],\n",
      "        [ 2.9288e-02, -3.4353e-01, -2.0979e-01,  3.5082e-01,  2.8308e-01,\n",
      "         -6.4974e-02,  2.7606e-02,  8.6001e-02],\n",
      "        [-1.3737e-01, -8.0567e-02, -1.1826e-01,  4.8684e-02,  2.5511e-01,\n",
      "          2.7370e-01,  3.0614e-01,  3.0791e-01],\n",
      "        [-2.8899e-01,  2.8429e-01, -2.0693e-01,  2.4925e-01, -1.7135e-02,\n",
      "         -1.4282e-01,  2.8891e-01, -7.2847e-02],\n",
      "        [-1.1045e-01, -1.0121e-01, -1.8394e-01, -1.6141e-02,  5.2969e-02,\n",
      "          2.2252e-01,  7.2905e-02, -1.0540e-01]], requires_grad=True)\n",
      "tensor([True, True, True, True, True, True, True, True])\n"
     ]
    }
   ],
   "source": [
    "# 我们需要给共享层一个名称，以便可以引用它的参数。\n",
    "shared = nn.Linear(8, 8)\n",
    "net = nn.Sequential(nn.Linear(4, 8), nn.ReLU(),\n",
    "                    shared, nn.ReLU(),\n",
    "                    shared, nn.ReLU(),\n",
    "                    nn.Linear(8, 1))\n",
    "print(net)\n",
    "net(X)\n",
    "# 检查参数是否相同\n",
    "print(net[2].weight.data[0] == net[4].weight.data[0])\n",
    "net[2].weight.data[0, 0] = 101\n",
    "print(net[2].weight)\n",
    "# 确保它们实际上是同一个对象，而不只是有相同的值。\n",
    "print(net[2].weight.data[0] == net[4].weight.data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "590f730c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
